{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "%matplotlib notebook"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "import datetime as dt\n",
    "import os\n",
    "import glob\n",
    "import calendar\n",
    "import re"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def read_precip(fili):\n",
    "    \"\"\"\n",
    "    Reader for precipitation files contained in the NPSNOW data set\n",
    "\n",
    "    In NPSNOW precip files, dry-days have amount=-9.9 and type=-9.\n",
    "    These are set to 0.0 and 0 respectively.  This dry-days are distinct\n",
    "    from days with trace precipitation, which have amount 0.0 and non-zero \n",
    "    type.\n",
    "\n",
    "    Arguments\n",
    "    ---------\n",
    "    fili - file path\n",
    "\n",
    "    Returns\n",
    "    -------\n",
    "    Pandas dataframe containg precipitation data for one station\n",
    "    \"\"\"\n",
    "\n",
    "    df = pd.read_csv(fili, header=None, delim_whitespace=True,\n",
    "                     #na_values={'amount': -9.9, 'type': -9}, \n",
    "                     names=['statid','month','day','year','amount','type'])\n",
    "\n",
    "    isday = [row[1]['day'] <= \\\n",
    "             calendar.monthrange( int(row[1]['year']),int(row[1]['month']) )[1] \\\n",
    "             for row in df.iterrows()]\n",
    "    df = df[isday] # only return rows with valid date\n",
    "\n",
    "    df.index = [dt.datetime(int(row[1]['year']),\n",
    "                            int(row[1]['month']),\n",
    "                            int(row[1]['day']) ) \\\n",
    "                for row in df.iterrows()] # Reset index to date\n",
    "\n",
    "    # Assumes zero precipitation/dry days are marked as -9.9, set to zero\n",
    "    df = df.where(df > 0., 0.0)\n",
    "    \n",
    "    return df[['statid','amount','type']]\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Get files containing precipitation.  Following Bogdanova et al (2006), I exclude NP 4, 5 and 14"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "dirpath = r'C:\\Users\\apbarret\\Documents\\data\\SnowOnSeaIce\\NPSNOW\\precip'\n",
    "filelist = glob.glob(os.path.join(dirpath,'np_??_??.pre'))\n",
    "filelist = [f for f in filelist if not re.search('np_03|np_04|np_14',f)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>statid</th>\n",
       "      <th>amount</th>\n",
       "      <th>type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1955-05-01</th>\n",
       "      <td>5</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1955-05-02</th>\n",
       "      <td>5</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1955-05-03</th>\n",
       "      <td>5</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1955-05-04</th>\n",
       "      <td>5</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1955-05-05</th>\n",
       "      <td>5</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "            statid  amount  type\n",
       "1955-05-01       5     0.0     0\n",
       "1955-05-02       5     0.0     0\n",
       "1955-05-03       5     0.0     0\n",
       "1955-05-04       5     0.0     0\n",
       "1955-05-05       5     0.0     0"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.concat([read_precip(f) for f in filelist])\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th>statid</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "      <th>8</th>\n",
       "      <th>9</th>\n",
       "      <th>10</th>\n",
       "      <th>11</th>\n",
       "      <th>12</th>\n",
       "      <th>13</th>\n",
       "      <th>15</th>\n",
       "      <th>...</th>\n",
       "      <th>22</th>\n",
       "      <th>23</th>\n",
       "      <th>24</th>\n",
       "      <th>25</th>\n",
       "      <th>26</th>\n",
       "      <th>27</th>\n",
       "      <th>28</th>\n",
       "      <th>29</th>\n",
       "      <th>30</th>\n",
       "      <th>31</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1955-05-01</th>\n",
       "      <td>1.3</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1955-06-01</th>\n",
       "      <td>7.2</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1955-07-01</th>\n",
       "      <td>13.7</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1955-08-01</th>\n",
       "      <td>27.6</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1955-09-01</th>\n",
       "      <td>12.3</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 26 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "statid        5   6   7   8   9   10  11  12  13  15 ...  22  23  24  25  26  \\\n",
       "1955-05-01   1.3 NaN NaN NaN NaN NaN NaN NaN NaN NaN ... NaN NaN NaN NaN NaN   \n",
       "1955-06-01   7.2 NaN NaN NaN NaN NaN NaN NaN NaN NaN ... NaN NaN NaN NaN NaN   \n",
       "1955-07-01  13.7 NaN NaN NaN NaN NaN NaN NaN NaN NaN ... NaN NaN NaN NaN NaN   \n",
       "1955-08-01  27.6 NaN NaN NaN NaN NaN NaN NaN NaN NaN ... NaN NaN NaN NaN NaN   \n",
       "1955-09-01  12.3 NaN NaN NaN NaN NaN NaN NaN NaN NaN ... NaN NaN NaN NaN NaN   \n",
       "\n",
       "statid      27  28  29  30  31  \n",
       "1955-05-01 NaN NaN NaN NaN NaN  \n",
       "1955-06-01 NaN NaN NaN NaN NaN  \n",
       "1955-07-01 NaN NaN NaN NaN NaN  \n",
       "1955-08-01 NaN NaN NaN NaN NaN  \n",
       "1955-09-01 NaN NaN NaN NaN NaN  \n",
       "\n",
       "[5 rows x 26 columns]"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Generate table organized by index time and station\n",
    "table = pd.pivot_table(df, values='amount', index=df.index, columns='statid')\n",
    "table = table.resample('MS').sum() # Generate month sums\n",
    "table = table.where(table > 0) # Set zero to NaN\n",
    "table.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Parch</th>\n",
       "      <th>n</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1956-12-31</th>\n",
       "      <td>158.300000</td>\n",
       "      <td>16.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1957-12-31</th>\n",
       "      <td>177.650000</td>\n",
       "      <td>20.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1958-12-31</th>\n",
       "      <td>149.200000</td>\n",
       "      <td>24.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1962-12-31</th>\n",
       "      <td>121.550000</td>\n",
       "      <td>23.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1963-12-31</th>\n",
       "      <td>125.850000</td>\n",
       "      <td>24.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1964-12-31</th>\n",
       "      <td>119.600000</td>\n",
       "      <td>24.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1965-12-31</th>\n",
       "      <td>104.250000</td>\n",
       "      <td>16.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1966-12-31</th>\n",
       "      <td>174.700000</td>\n",
       "      <td>20.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1967-12-31</th>\n",
       "      <td>135.450000</td>\n",
       "      <td>16.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1969-12-31</th>\n",
       "      <td>190.283333</td>\n",
       "      <td>26.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1970-12-31</th>\n",
       "      <td>164.450000</td>\n",
       "      <td>33.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1971-12-31</th>\n",
       "      <td>127.550000</td>\n",
       "      <td>39.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1972-12-31</th>\n",
       "      <td>191.600000</td>\n",
       "      <td>28.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1973-12-31</th>\n",
       "      <td>171.500000</td>\n",
       "      <td>18.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1974-12-31</th>\n",
       "      <td>187.400000</td>\n",
       "      <td>16.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1975-12-31</th>\n",
       "      <td>126.200000</td>\n",
       "      <td>12.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1976-12-31</th>\n",
       "      <td>161.050000</td>\n",
       "      <td>23.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1977-12-31</th>\n",
       "      <td>154.300000</td>\n",
       "      <td>24.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1978-12-31</th>\n",
       "      <td>125.283333</td>\n",
       "      <td>29.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1979-12-31</th>\n",
       "      <td>177.850000</td>\n",
       "      <td>24.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1980-12-31</th>\n",
       "      <td>156.550000</td>\n",
       "      <td>22.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1981-12-31</th>\n",
       "      <td>138.700000</td>\n",
       "      <td>19.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1982-12-31</th>\n",
       "      <td>155.050000</td>\n",
       "      <td>15.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1983-12-31</th>\n",
       "      <td>180.400000</td>\n",
       "      <td>19.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1984-12-31</th>\n",
       "      <td>152.950000</td>\n",
       "      <td>21.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1985-12-31</th>\n",
       "      <td>78.400000</td>\n",
       "      <td>12.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1986-12-31</th>\n",
       "      <td>111.750000</td>\n",
       "      <td>19.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1987-12-31</th>\n",
       "      <td>135.533333</td>\n",
       "      <td>24.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1988-12-31</th>\n",
       "      <td>177.916667</td>\n",
       "      <td>29.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1989-12-31</th>\n",
       "      <td>190.350000</td>\n",
       "      <td>24.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1990-12-31</th>\n",
       "      <td>117.750000</td>\n",
       "      <td>24.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                 Parch     n\n",
       "1956-12-31  158.300000  16.0\n",
       "1957-12-31  177.650000  20.0\n",
       "1958-12-31  149.200000  24.0\n",
       "1962-12-31  121.550000  23.0\n",
       "1963-12-31  125.850000  24.0\n",
       "1964-12-31  119.600000  24.0\n",
       "1965-12-31  104.250000  16.0\n",
       "1966-12-31  174.700000  20.0\n",
       "1967-12-31  135.450000  16.0\n",
       "1969-12-31  190.283333  26.0\n",
       "1970-12-31  164.450000  33.0\n",
       "1971-12-31  127.550000  39.0\n",
       "1972-12-31  191.600000  28.0\n",
       "1973-12-31  171.500000  18.0\n",
       "1974-12-31  187.400000  16.0\n",
       "1975-12-31  126.200000  12.0\n",
       "1976-12-31  161.050000  23.0\n",
       "1977-12-31  154.300000  24.0\n",
       "1978-12-31  125.283333  29.0\n",
       "1979-12-31  177.850000  24.0\n",
       "1980-12-31  156.550000  22.0\n",
       "1981-12-31  138.700000  19.0\n",
       "1982-12-31  155.050000  15.0\n",
       "1983-12-31  180.400000  19.0\n",
       "1984-12-31  152.950000  21.0\n",
       "1985-12-31   78.400000  12.0\n",
       "1986-12-31  111.750000  19.0\n",
       "1987-12-31  135.533333  24.0\n",
       "1988-12-31  177.916667  29.0\n",
       "1989-12-31  190.350000  24.0\n",
       "1990-12-31  117.750000  24.0"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "month_table = pd.DataFrame({'Parch': table.mean(axis=1), 'n': table.count(axis=1)})\n",
    "month_table.resample('Y').sum(min_count=12).dropna()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1955-12-31     8\n",
       "1956-12-31    16\n",
       "1957-12-31    20\n",
       "1958-12-31    24\n",
       "1959-12-31    11\n",
       "1960-12-31     8\n",
       "1961-12-31    13\n",
       "1962-12-31    23\n",
       "1963-12-31    24\n",
       "1964-12-31    24\n",
       "1965-12-31    16\n",
       "1966-12-31    20\n",
       "1967-12-31    16\n",
       "1968-12-31    19\n",
       "1969-12-31    26\n",
       "1970-12-31    33\n",
       "1971-12-31    39\n",
       "1972-12-31    28\n",
       "1973-12-31    18\n",
       "1974-12-31    16\n",
       "1975-12-31    12\n",
       "1976-12-31    23\n",
       "1977-12-31    24\n",
       "1978-12-31    29\n",
       "1979-12-31    24\n",
       "1980-12-31    22\n",
       "1981-12-31    19\n",
       "1982-12-31    15\n",
       "1983-12-31    19\n",
       "1984-12-31    21\n",
       "1985-12-31    12\n",
       "1986-12-31    19\n",
       "1987-12-31    24\n",
       "1988-12-31    29\n",
       "1989-12-31    24\n",
       "1990-12-31    24\n",
       "1991-12-31     5\n",
       "Freq: A-DEC, dtype: int64"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "table.count(axis=1).resample('Y').sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
